package com.eascs.spider.yuqing.worker.spider;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.eascs.spider.yuqing.common.proj.Platform;
import com.eascs.spider.yuqing.common.proj.Task;
import com.eascs.spider.yuqing.common.util.Charset;
import com.eascs.spider.yuqing.common.util.NetUtil;
import com.eascs.spider.yuqing.worker.util.MongodbUtil;

/**
 * 新浪新闻spider
 * @author hzp
 *
 */
public class SinaNewsSpider extends SimpleSpider{

	private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM");
	private static SimpleDateFormat sdf_sina = new SimpleDateFormat("yyyy年MM月dd日HH:mm");
	private String time;											 
	public SinaNewsSpider(String url) {
		super(url);
		this.time = sdf.format(new Date(System.currentTimeMillis()));
	}

	@Override
	public void run(){
		try {
			//判断是否当月的新闻或首页
			if(!url.contains(time) && !url.equals("http://news.sina.com.cn/"))
				return;
			Map<String, String> header = new HashMap<String, String>();
			header.put("Host", "news.sina.com.cn");
			//新闻专栏部分字符集为gb2312，其他默认utf-8
			String charset = url.contains("http://news.sina.com.cn/zl")?Charset.CHARSET_GB2312:Charset.UTF_8;
			String html = NetUtil.loadPageRawTextByGet(url, client, header,null, charset);
			if(html == null){
				logger.info("url：{}请求返回为空" ,url);
				return;
			}
			extractUrl(html);
			//判断是否新浪新闻主页
			if(!url.equals("http://news.sina.com.cn/"))
				extractInfo(html);
		} catch (Exception e) {
			logger.error("url：{}爬取异常" ,url , e);
		}
	}
	
	/**
	 * 从html源码中提取标题，内容，时间，来源
	 * @param html
	 * @return
	 * @throws ParseException 
	 */
	public void extractInfo(String html) throws ParseException{
		Map<String, Object> data = new HashMap<String, Object>();
		Document document = Jsoup.parse(html);
		//新版新浪新闻以两个h1标签为依据
		if(document.getElementsByTag("h1").size()==2){
			extractInfoNew(html);
			return;
		}
		
		String artibodyTitle = document.getElementById("artibodyTitle").text();
		logger.info("标题：{}" ,artibodyTitle);
		data.put("title", artibodyTitle);
		String artibody = document.getElementById("artibody").text();
		logger.info("内容：{}" ,artibody);
		data.put("content", artibody);
		//1,默认取时间及作者方式
		if(document.getElementById("navtimeSource") != null){
			String navtimeSource = document.getElementById("navtimeSource").text();
			if(navtimeSource.split(" ").length == 2){
				logger.info("时间：{},来源：{}" ,navtimeSource.split(" ")[0] ,navtimeSource.split(" ")[1]);
				data.put("time", sdf_sina.parse(navtimeSource.split(" ")[0].replace(" ", "")));
				data.put("source", navtimeSource.split(" ")[1]);
			}else{
				logger.info("时间：{}" ,sdf_sina.parse(navtimeSource));
				data.put("time", sdf_sina.parse(navtimeSource.replace(" ", "")));
			}
		}else if(document.getElementById("pub_date") != null){
			//2,新闻锐见等博客类
			String time = document.getElementById("pub_date").text();
			String author = document.getElementById("author_ename").text();
			logger.info("时间：{},作者：{}" ,time , author);
			data.put("time", sdf_sina.parse(time.replace(" ", "")));
			data.put("author", author);
		}
		data.put("url", url);
		data.put("crawler_time", new Date(System.currentTimeMillis()));
		MongodbUtil.setYuqing(data, Platform.SINA_NEWS);
	}
	
	public void extractInfoNew(String html) throws ParseException{
		Document document = Jsoup.parse(html);
		String title = document.selectFirst("h1.main-title").text();
		String time = document.selectFirst("div#top_bar span.date").text();
		String source = document.selectFirst("div#top_bar div.date-source a").text();
		String content = document.getElementById("article_content").text();
		Map<String, Object> data = new HashMap<>();
		data.put("title", title);
		data.put("time", sdf_sina.parse(time.replace(" ", "")));
		data.put("source", source);
		data.put("content", content);
		logger.info("data:{}" ,data.toString());
		data.put("url", url);
		data.put("crawler_time", new Date(System.currentTimeMillis()));
		MongodbUtil.setYuqing(data, Platform.SINA_NEWS);
	}
	
	/**
	 * 从html源码中提取符合新浪新闻链接格式的url
	 * @param html
	 * @return
	 */
	public void extractUrl(String html){
		Pattern pattern = Pattern.compile("http://news.sina.com.cn.*?htm(?:l)");
		Matcher matcher = pattern.matcher(html);
		while(matcher.find()){
			String url = matcher.group();
			//过滤不符合条件的url,以及不是当月的url
			if(url.contains("\"") || url.contains("{") || !url.contains(time))
				continue;
			Task task = new Task();
			task.setUrl(url);
			manager.addSendQueue(task);
		}
	}
	
	public static void main(String[] args) {
		SinaNewsSpider spider = new SinaNewsSpider("http://news.sina.com.cn/c/nd/2017-12-22/doc-ifypxrpp3363783.shtml");
		Thread thread = new Thread(spider);
		thread.start();
	}
}
